package crawler;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
import java.util.UUID;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import util.DAO;
import util.Properties;



public class Crawler extends WebCrawler {
		
		private List<String> whiteList;
		private List<String> blackList;
		private String blackPattern;
		private List<Map<String,String>> fields;
		private String bodySelector;
		private String storeId;
		private String crawlId;
		private DAO dao;
		
		//private int debuger;
		
		private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g" 
            + "|png|tiff?|mid|mp2|mp3|mp4"
            + "|wav|avi|mov|mpeg|ram|m4v|pdf" 
            + "|rm|smil|wmv|swf|wma|zip|rar|gz|ico))$");

		@Override
		public void onStart(){
			
            Properties prop = (Properties) this.getMyController().getCustomData();
            
            storeId = prop.getString("storeId");
            whiteList = prop.getList("seed");
            blackList = prop.getList("disallow");
            blackPattern = prop.getString("disallow_pattern");
            bodySelector = prop.getString("body_selector");
            crawlId = prop.getString("crawlId");
            
			String db_host = prop.getString("db_host");
			String db_name = prop.getString("db_name");
            String db_user = prop.getString("db_user");
            String db_pass = prop.getString("db_pass");
            
            dao = new DAO(db_host, db_name, db_user, db_pass);
            fields = dao.getStoreFields(storeId);
            
		}
		/**
         * You should implement this function to specify whether the given url
         * should be crawled or not (based on your crawling logic).
         */
        @Override
        public boolean shouldVisit(WebURL url) {
                String href = url.getURL().toLowerCase();
                
                if (FILTERS.matcher(href).matches()) return false;
                for(String blackUrl : blackList){
                	if(!blackUrl.isEmpty() && href.startsWith(blackUrl)) return false;
                }
                if(blackPattern != null && !blackPattern.isEmpty() && href.matches(blackPattern)){
                	return false;
                }
                for(String whiteUrl : whiteList){
                	if(href.startsWith(whiteUrl))	return true;
                }
                //if not in white list don't visit
                return false;
        }

        /**
         * This function is called when a page is fetched and ready to be processed
         * by your program.
         */
        @Override
        public void visit(Page page) {
        		String url = page.getWebURL().getURL();
        		
        		String itemId = UUID.randomUUID().toString();
        		
                if (page.getParseData() instanceof HtmlParseData) {
                    HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
                    String html = htmlParseData.getHtml();
                    Document doc = Jsoup.parse(html);
                    
                	try {
                		Elements body = doc.select(bodySelector);
                        if(body == null || body.isEmpty()) throw new Exception();
                        String name = "";
                		for(Map<String,String> field : fields){
                			
                			String fieldId = field.get("field_id");
                			boolean mandatory = "Y".equals(field.get("mandatory"));
                			String defaultValue = field.get("default")==null ? "" : field.get("default");
            				
                			//Do we have a selector?
                			String selector = (String) field.get("selector");
            				if(selector.isEmpty()){
            					if(mandatory) throw new Exception();
            					if(!defaultValue.isEmpty()) dao.addField(crawlId, itemId, fieldId, defaultValue);
            					continue;
            				}
            				
            				//Do we have a DOM element for that selector 
                			Element element = doc.select(selector).first();
                			if(element==null){
                				if(mandatory) throw new Exception();
                				if(!defaultValue.isEmpty()) dao.addField(crawlId, itemId, fieldId, defaultValue);
            					continue;
                			}
                			
                			//Validate and transform the element's content
                			String strValue = "img".equals(element.tagName()) ? absoluteURL(url, element.attr("src")) : element.text();
                			if(strValue == null || strValue.isEmpty()){
                				if(mandatory) throw new Exception();
                				if(!defaultValue.isEmpty()) dao.addField(crawlId, itemId, fieldId,defaultValue);
                				continue;
                			}
                			if("float".equals(field.get("type"))){
                				String digits  = digitize(strValue);
                				if(digits==null){
                					if (mandatory) throw new Exception();
                					if(!defaultValue.isEmpty()) dao.addField(crawlId, itemId, fieldId,defaultValue);
                				}
                				dao.addField(crawlId, itemId, fieldId, digits);
                			}else{
                				dao.addField(crawlId, itemId, fieldId, strValue);
                			}
                			if("name".equals(fieldId)) name  = strValue;
                		}
                		dao.addItem(itemId, storeId, crawlId, name, url);
						
					} catch (Exception e) {
						
					}
                }
                //format = new SimpleDateFormat("hh:mm:ss");
                //System.out.println(format.format(new Date())+" ("+this.myController.getFrontier().getNumberOfProcessedPages()+","+hits+","+debuger+") "+url);
        }
        
        private String digitize(String value){
        	
        	if(value == null || value.isEmpty()) return null;
        	
        	String digits = new String();
        	boolean dot = true;
        	
        	for (int i=value.length()-1;i>=0;i--){
        		char c = value.charAt(i);
        		if(c >= '0' && c <= '9' ) digits = c+digits;
        		else if((c == '.' || c == ',') && dot){
        			digits = '.'+digits;
        			dot = false;
        		}
        	}
        	return digits;
        }
        private String absoluteURL(String base, String relative ){
        	if(relative==null || relative.isEmpty() || relative.startsWith("http"))
        		return relative;
			if(relative.startsWith("/") || relative.startsWith("\\")){
				try {
					URL pageurl = new URL(base);
					return pageurl.getProtocol()+"://"+pageurl.getHost()+relative;
				} catch (MalformedURLException e) {
					return "MalformedURLException/"+relative;
				}
			}else{
				int slash = base.lastIndexOf("/");
				if(slash<0) slash = base.lastIndexOf("\\");
				if(slash<0) return base+"/"+relative;
				
				return base.substring(0,slash)+"/"+relative;
			}
        }
        
        @Override
        public void onBeforeExit() {
        	
        }
}